
from langchain.callbacks import streaming_stdout
from langchain import HuggingFaceTextGenInference


class langchaindemo:
    def api_text_generation(self):
        """
        # 安装huggingface的text_generation,这是一个客户端。事实上，需要安装一个服务端才能够真正与之通信
        # 目前还没有成功，因为安装非常复杂
        # 我在等23H2更新后,才能够在wsl中试验成功
        # pip install text_generation
        # python server.py --api
        """

        callbacks = [streaming_stdout.StreamingStdOutCallbackHandler()]
        llm = HuggingFaceTextGenInference(
            inference_server_url="http://127.0.0.1:5000/",
            max_new_tokens=512,
            top_k=10,
            top_p=0.95,
            typical_p=0.95,
            temperature=0.01,
            repetition_penalty=1.03,
            callbacks=callbacks
        )

        print(llm("What is Deep Learning?"))

